package reflection;

import org.apache.http.HttpEntity;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * User:bob.yang
 * Date 2015/8/24 0024
 */
public class TestMain {

    //所有的地址
    public static List<String> urls = new ArrayList<>();
    //待访问的地址
    public static List<String> goUrls = new ArrayList<>();
    //目标的地址
    public static List<String> goalUrls = new ArrayList<>();


    public static void main(String[] args) {
        RequestConfig defaultRequestConfig = RequestConfig.custom()
                .setSocketTimeout(5000)
                .setConnectTimeout(5000)
                .setConnectionRequestTimeout(5000)
                .setCookieSpec(CookieSpecs.STANDARD_STRICT)
                .build();
        CloseableHttpClient httpClient = HttpClients.custom()
                .setDefaultRequestConfig(defaultRequestConfig)
                .build();
//        CloseableHttpClient httpClient = HttpClients.createDefault();
//        httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10000);


//        HttpGet httpGet =   new HttpGet("http://book.douban.com/subject/26392727/?icn=index-editionrecommend");
//        CloseableHttpResponse closeableHttpResponse = httpClient.execute(httpGet);
//        HttpEntity entity = closeableHttpResponse.getEntity();
//
//        Document doc = Jsoup.parse(EntityUtils.toString(entity));
//        Elements elements = doc.select("a[href]");
//        for(Element node : elements){
//            String href = node.attr("href");
//            String text = node.text();
//            System.out.println(text + ":" + href);
//        }
//        closeableHttpResponse.close();
//
        goUrls.add("http://book.douban.com/");
        while(true) {
            if(goalUrls.size() > 2000) {
                break;
            }
            if(goUrls.isEmpty()) {
                break;
            }
            List<String> tempList = new ArrayList<>();
            for(String url : goUrls) {
                tempList.add(url);
            }
            goUrls.clear();
            for(String url : tempList) {
                System.out.println("地址："+url);
                urls.add(url);

                URL url1 = null;
                try {
                    url1 = new URL(url);
                } catch (MalformedURLException e) {
                   continue;
                }
                URI uri =null;
                try {
                    uri = new URI(url1.getProtocol(), url1.getHost(), url1.getPath(), url1.getQuery(), null);
                } catch (URISyntaxException e) {
                   continue;
                }
                if(uri == null) {
                    continue;
                }
                HttpGet httpGet =   new HttpGet(uri);
                httpGet.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
                httpGet.setHeader("Accept-Encoding","gzip,deflate,sdch");
                httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.8");
                httpGet.setHeader("Connection","keep-alive");
                httpGet.setHeader("Cache-Control","max-age=0");
//                httpGet.setHeader("Host","movie.douban.com");
                httpGet.setHeader("Cookie", "bid=\"vsXMg4Bay30\"; ll=\"108296\"; _ga=GA1.2.1038398848.1430800075; ps=y; viewed=\"26569724_26390842_26451423_5975576_26392727_25773187_26413027_4010969_24722612_26274206\"; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1440396583%2C%22http%3A%2F%2Faccounts.douban.com%2Flogin%3Fuid%3D51295828%26alias%3D1245964132%2540qq.com%26redir%3Dhttp%253A%252F%252Fwww.douban.com%252F%26source%3Dindex_nav%26error%3D1013%22%5D; ue=\"1245964132@qq.com\"; dbcl2=\"51295828:5qKM2LBoMeY\"; ck=\"YWLZ\"; ap=1; push_noty_num=0; push_doumail_num=0; _pk_id.100001.8cb4=84049da8a64e0f42.1438736478.36.1440407338.1440392367.; _pk_ses.100001.8cb4=*; __utma=30149280.1038398848.1430800075.1440391513.1440396243.37; __utmb=30149280.68.10.1440396243; __utmc=30149280; __utmz=30149280.1439520983.20.8.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmv=30149280.5129");
                httpGet.setHeader("Upgrade-Insecure-Requests", "1");
                httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36");
                CloseableHttpResponse closeableHttpResponse = null;
                try {
                    closeableHttpResponse = httpClient.execute(httpGet);
                    httpGet.setHeader("Referer", url);
                    System.out.println("状态：" +closeableHttpResponse.getStatusLine());
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                       System.out.print("线程有问题！");
                    }
                } catch (IOException e) {
                    System.out.println(e);
                    continue;
                }
                if(url.equals("http://www.douban.com/people/80274259/")) {
                    System.out.println("蛋疼");
                }
                HttpEntity entity = closeableHttpResponse.getEntity();
                Document doc = null;
                try {
                    doc = Jsoup.parse(EntityUtils.toString(entity));
                } catch (IOException e) {
                    System.out.println(e);
                    continue;
                }
                Elements elements = doc.select("a[href]");
                for(Element node : elements){
                    String href = node.attr("href");
                    String text = node.text();
                    solveUrl(href);
                }
                try {
                    closeableHttpResponse.close();
                } catch (IOException e) {
                    continue;
                }
            }
        }
        File file = new File("D://goUrl.text");
        FileWriter fileWriter = null;
        try {
            fileWriter = new FileWriter(file);
        } catch (IOException e) {
            e.printStackTrace();
        }
        for(String str : goalUrls) {
            try {
                fileWriter.write(str);
                fileWriter.write("\r\n");
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            fileWriter.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void solveUrl(String url) {
        //过滤不合法的url
        if("".equals(url) || !url.startsWith("http://") || !url.contains("douban")) {
            return;
        }
        if(!url.contains("www")) {
//            System.out.println("地址："+url);
            String temp = url.substring(url.indexOf("/")+2);
//            url = "http://www." + temp;
        }

        //判断这个url是不是目标url
        if(checkUrl(url)) {
            if(!goalUrls.contains(url)){
                goalUrls.add(url);
            }
        }
        //首先判断历史访问里有没有
        if(!urls.contains(url)) {
            //如果历史访问里面没有，就判断是否在待访问的里
            if(!goUrls.contains(url)) {
                goUrls.add(url);
            }
        }
    }

    public static boolean checkUrl(String uri) {
        return uri.startsWith("http://book.douban.com/subject/");
    }

}
